package com.hlmc.spider;
import com.hlmc.config.Constants;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;

@Service("rank_spider")
public class Rank_spider extends BaseProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(100).setCharset("utf-8" + "");

    @Override
    public void process(Page page) {



            String zy_p_title=page.getHtml().xpath("//div[@data-block='tj-zyrank']/div[@class='p-mod rank']/div[1]/span/text()").all().toString().replace("[","").replace("]","");
            String zy_v_title=page.getHtml().xpath("//div[@data-block='tj-zyrank']/div[@class='p-mod rank']/div[2]/ul/li/div/a/text()").all().toString().replace("[","").replace("]","").replace(" ","");


            if( (zy_p_title == null)||zy_p_title.replace(" ", "").equals("")) {

                page.setSkip(true);
            }
            System.out.println(zy_p_title);
            System.out.println(zy_v_title);


        String tv_p_title=page.getHtml().xpath("//div[@data-block='tj-dsrank']/div[@class='p-mod rank']/div[1]/span/text()").all().toString().replace("[","").replace("]","");
        String tv_v_title=page.getHtml().xpath("//div[@data-block='tj-dsrank']/div[@class='p-mod rank']/div[2]/ul/li/div/a/text()").all().toString().replace("[","").replace("]","").replace(" ","");
        if( (tv_p_title == null)||tv_p_title.replace(" ", "").equals("")) {

            page.setSkip(true);
        }
        System.out.println(tv_p_title);
        System.out.println(tv_v_title);

        String dy_p_title=page.getHtml().xpath("//div[@data-block='tj-dyrank']/div[@class='p-mod rank']/div[1]/span/text()").all().toString().replace("[","").replace("]","");
        String dy_v_title=page.getHtml().xpath("//div[@data-block='tj-dyrank']/div[@class='p-mod rank']/div[2]/ul/li/div/a/text()").all().toString().replace("[","").replace("]","").replace(" ","");
        if( (dy_p_title == null)||dy_p_title.replace(" ", "").equals("")) {

            page.setSkip(true);
        }
        System.out.println(dy_p_title);
        System.out.println(dy_v_title);


        String dm_p_title=page.getHtml().xpath("//div[@data-block='tj-dmrank']/div[@class='p-mod rank']/div[1]/span/text()").all().toString().replace("[","").replace("]","");
        String dm_v_title=page.getHtml().xpath("//div[@data-block='tj-dmrank']/div[@class='p-mod rank']/div[2]/ul/li/div/a/text()").all().toString().replace("[","").replace("]","").replace(" ","");
        if( (dm_p_title == null)||dm_p_title.replace(" ", "").equals("")) {

            page.setSkip(true);
        }
        System.out.println(dm_p_title);
        System.out.println(dm_v_title);


        page.putField("dm_p_title", dm_p_title);
        page.putField("dm_v_title", dm_v_title);
        page.putField("tv_p_title", tv_p_title);
        page.putField("tv_v_title", tv_v_title);
        page.putField("dy_p_title", dy_p_title);
        page.putField("dy_v_title", dy_v_title);
        page.putField("zy_p_title", zy_p_title);
        page.putField("zy_v_title", zy_v_title);


    }

    @Override
    public Site getSite() {
        return site;
    }


    public static void main(String[] args){
        Spider.create(new Rank_spider())
                .addUrl("https://www.360kan.com/")
                .addPipeline(new Rank_pipeline())
                .thread(5)
                .run();

    }

}
